Automated web scraping using Obsidian Web Clipper and Puppeteer.

"content_scripts": [
		{
			"js": ["scripts/content.js"],
			// Make sure to update this array to include any URLs you
			// want to scrape
			"matches": ["https://developer.chrome.com/docs/extensions/*",
			 "https://developer.chrome.com/docs/webstore/*"]
		}
	]

// This needs to be set to the id of the extension you want to trigger
const const id: "lemfefnbebfkbajcafkjoklibjadafasg"id = 'lemfefnbebfkbajcafkjoklibjadafasg';

browser.runtime.onMessageExternal.addListener(
	(
		request: unknownrequest: unknown,
		sender: browser.Runtime.MessageSendersender: browser.Runtime.type browser.Runtime.MessageSender = /*unresolved*/ anyMessageSender,
		sendResponse: (response?: any) => voidsendResponse: (response: anyresponse?: any) => void
	): true | undefined => {
		if (typeof request: unknownrequest === "object" && request: object | nullrequest !== null) {
			const 
const typedRequest: {
    action: string;
    isActive?: boolean | undefined;
    hasHighlights?: boolean | undefined;
    tabId?: number | undefined;
}typedRequest = request: objectrequest as {
				action: stringaction: string;
				isActive?: boolean | undefinedisActive?: boolean;
				hasHighlights?: boolean | undefinedhasHighlights?: boolean;
				tabId?: number | undefinedtabId?: number;
			};

			
			if (
const typedRequest: {
    action: string;
    isActive?: boolean | undefined;
    hasHighlights?: boolean | undefined;
    tabId?: number | undefined;
}typedRequest.action: stringaction === "triggerQuickClip") {
				browser.tabs
					.query({ active: booleanactive: true, currentWindow: booleancurrentWindow: true })
					.then((tabs: anytabs) => {
						if (tabs: anytabs[0]?.id) {
							browser.action.openPopup();
							function setTimeout<[]>(callback: () => void, ms?: number | undefined): NodeJS.Timeout (+2 overloads)
Schedules execution of a one-time `callback` after `delay` milliseconds.

The `callback` will likely not be invoked in precisely `delay` milliseconds.
Node.js makes no guarantees about the exact timing of when callbacks will fire,
nor of their ordering. The callback will be called as close as possible to the
time specified.

When `delay` is larger than `2147483647` or less than `1`, the `delay`will be set to `1`. Non-integer delays are truncated to an integer.

If `callback` is not a function, a `TypeError` will be thrown.

This method has a custom variant for promises that is available using `timersPromises.setTimeout()`.
@sincev0.0.1@paramcallback The function to call when the timer elapses.@paramdelay The number of milliseconds to wait before calling the `callback`.@paramargs Optional arguments to pass when the `callback` is called.@returnfor use with {@link clearTimeout}setTimeout(() => {
								browser.runtime
									.sendMessage({ action: stringaction: "triggerQuickClip" })
									.catch((error: anyerror) =>
										sendResponse: (response?: any) => voidsendResponse({ success: anysuccess: error: anyerror })
									);
							}, 500);
						}
					});
				return true;
			}

			// For other actions that use sendResponse
			if (
				
const typedRequest: {
    action: string;
    isActive?: boolean | undefined;
    hasHighlights?: boolean | undefined;
    tabId?: number | undefined;
}typedRequest.action: stringaction === "extractContent" ||
				
const typedRequest: {
    action: string;
    isActive?: boolean | undefined;
    hasHighlights?: boolean | undefined;
    tabId?: number | undefined;
}typedRequest.action: stringaction === "ensureContentScriptLoaded" ||
				
const typedRequest: {
    action: string;
    isActive?: boolean | undefined;
    hasHighlights?: boolean | undefined;
    tabId?: number | undefined;
}typedRequest.action: stringaction === "getHighlighterMode" ||
				
const typedRequest: {
    action: string;
    isActive?: boolean | undefined;
    hasHighlights?: boolean | undefined;
    tabId?: number | undefined;
}typedRequest.action: stringaction === "toggleHighlighterMode"
			) {
				return true;
			}
		}
		return var undefinedundefined;
	}
);

const asyncWait = async (time) => {
	await new Promise((resolve) => setTimeout(resolve, time))
};

(async () => {


	// MAC: /Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222 --no-first-run --no-default-browser-check --user-data-dir=$(mktemp -d -t 'chrome-remote_data_dir')
	
	// PC: start chrome.exe –remote-debugging-port=9222
	
	// Note: this url changes each time the command is run.
	const wsChromeEndpointUrl = 'YOUR_URL_HERE'
	const browser = await puppeteer.connect({
		browserWSEndpoint: wsChromeEndpointUrl,
	})

	for (let i = 0; i < cleanedLinks.length; i++) {
		const page = await browser.newPage()
		console.log(i + cleanedLinks[i])
		await page.goto(cleanedLinks[i], {
			waitUntil: 'domcontentloaded', 
		})
		await asyncWait(2000)

		await asyncWait(Math.random() * 1000)
		await page.close()
	}

	return
})();